{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "from torch.utils.data import DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x7fe7d407a770>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_epochs = 3\n",
    "batch_size_train = 64\n",
    "batch_size_test = 1000\n",
    "learning_rate = 0.01\n",
    "momentum = 0.5\n",
    "log_interval = 10\n",
    "random_seed = 1\n",
    "torch.manual_seed(random_seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loader = torch.utils.data.DataLoader(\n",
    "  torchvision.datasets.MNIST('./data/', train=True, download=True,\n",
    "                             transform=torchvision.transforms.Compose([\n",
    "                               torchvision.transforms.ToTensor(),\n",
    "                               torchvision.transforms.Normalize(\n",
    "                                 (0.1307,), (0.3081,))\n",
    "                             ])),\n",
    "  batch_size=batch_size_train, shuffle=True)\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(\n",
    "  torchvision.datasets.MNIST('./data/', train=False, download=True,\n",
    "                             transform=torchvision.transforms.Compose([\n",
    "                               torchvision.transforms.ToTensor(),\n",
    "                               torchvision.transforms.Normalize(\n",
    "                                 (0.1307,), (0.3081,))\n",
    "                             ])),\n",
    "  batch_size=batch_size_test, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NeuralNet(nn.Module):\n",
    "    \"\"\"\n",
    "    定义了一个784*2000*180*140*10的网络\n",
    "    \"\"\"\n",
    "    def __init__(self):\n",
    "        super(NeuralNet, self).__init__()\n",
    "        self.fc1 = nn.Linear(784, 2000) \n",
    "        self.relu = nn.ReLU()\n",
    "        self.fc2 = nn.Linear(2000, 180)\n",
    "        self.fc3 = nn.Linear(180, 140)\n",
    "        self.fc4 = nn.Linear(140, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        out = self.fc1(x)\n",
    "        out = self.relu(out)\n",
    "        out = self.fc2(out)\n",
    "        out = self.relu(out)\n",
    "        out = self.fc3(out)\n",
    "        out = self.relu(out)\n",
    "        out = self.fc4(out)\n",
    "        return F.log_softmax(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "network = NeuralNet()\n",
    "optimizer = optim.SGD(network.parameters(), lr=learning_rate,\n",
    "                      momentum=momentum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "NeuralNet(\n",
       "  (fc1): Linear(in_features=784, out_features=2000, bias=True)\n",
       "  (relu): ReLU()\n",
       "  (fc2): Linear(in_features=2000, out_features=180, bias=True)\n",
       "  (fc3): Linear(in_features=180, out_features=140, bias=True)\n",
       "  (fc4): Linear(in_features=140, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of paramerters in networks is 1956930  \n"
     ]
    }
   ],
   "source": [
    "print(\"Total number of paramerters in networks is {}  \".format(sum(x.numel() for x in network.parameters())))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_losses = []\n",
    "train_counter = []\n",
    "test_losses = []\n",
    "test_counter = [i*len(train_loader.dataset) for i in range(n_epochs + 1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([64, 1, 28, 28])\n",
      "torch.Size([64, 1, 28, 28])\n",
      "torch.Size([64])\n"
     ]
    }
   ],
   "source": [
    "for batch_idx, (data, target) in enumerate(train_loader):\n",
    "    print(data.size())\n",
    "    data.view(-1, 28*28)\n",
    "    print(data.size())\n",
    "    print(target.size())\n",
    "    \n",
    "    \n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-32-3c5581560232>:18: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  return F.log_softmax(out)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 1 [0/60000 (0%)]\tLoss: 2.300929\n",
      "Train Epoch: 1 [640/60000 (1%)]\tLoss: 2.277617\n",
      "Train Epoch: 1 [1280/60000 (2%)]\tLoss: 2.263643\n",
      "Train Epoch: 1 [1920/60000 (3%)]\tLoss: 2.247226\n",
      "Train Epoch: 1 [2560/60000 (4%)]\tLoss: 2.207326\n",
      "Train Epoch: 1 [3200/60000 (5%)]\tLoss: 2.157818\n",
      "Train Epoch: 1 [3840/60000 (6%)]\tLoss: 2.157007\n",
      "Train Epoch: 1 [4480/60000 (7%)]\tLoss: 2.095197\n",
      "Train Epoch: 1 [5120/60000 (9%)]\tLoss: 2.013647\n",
      "Train Epoch: 1 [5760/60000 (10%)]\tLoss: 1.951089\n",
      "Train Epoch: 1 [6400/60000 (11%)]\tLoss: 1.826463\n",
      "Train Epoch: 1 [7040/60000 (12%)]\tLoss: 1.569011\n",
      "Train Epoch: 1 [7680/60000 (13%)]\tLoss: 1.533198\n",
      "Train Epoch: 1 [8320/60000 (14%)]\tLoss: 1.350258\n",
      "Train Epoch: 1 [8960/60000 (15%)]\tLoss: 1.330681\n",
      "Train Epoch: 1 [9600/60000 (16%)]\tLoss: 1.124649\n",
      "Train Epoch: 1 [10240/60000 (17%)]\tLoss: 0.980358\n",
      "Train Epoch: 1 [10880/60000 (18%)]\tLoss: 0.788938\n",
      "Train Epoch: 1 [11520/60000 (19%)]\tLoss: 0.808089\n",
      "Train Epoch: 1 [12160/60000 (20%)]\tLoss: 0.725296\n",
      "Train Epoch: 1 [12800/60000 (21%)]\tLoss: 0.578540\n",
      "Train Epoch: 1 [13440/60000 (22%)]\tLoss: 0.843361\n",
      "Train Epoch: 1 [14080/60000 (23%)]\tLoss: 0.683771\n",
      "Train Epoch: 1 [14720/60000 (25%)]\tLoss: 0.631319\n",
      "Train Epoch: 1 [15360/60000 (26%)]\tLoss: 0.490487\n",
      "Train Epoch: 1 [16000/60000 (27%)]\tLoss: 0.616704\n",
      "Train Epoch: 1 [16640/60000 (28%)]\tLoss: 0.416383\n",
      "Train Epoch: 1 [17280/60000 (29%)]\tLoss: 0.481810\n",
      "Train Epoch: 1 [17920/60000 (30%)]\tLoss: 0.405137\n",
      "Train Epoch: 1 [18560/60000 (31%)]\tLoss: 0.589280\n",
      "Train Epoch: 1 [19200/60000 (32%)]\tLoss: 0.500204\n",
      "Train Epoch: 1 [19840/60000 (33%)]\tLoss: 0.314483\n",
      "Train Epoch: 1 [20480/60000 (34%)]\tLoss: 0.432849\n",
      "Train Epoch: 1 [21120/60000 (35%)]\tLoss: 0.376199\n",
      "Train Epoch: 1 [21760/60000 (36%)]\tLoss: 0.537921\n",
      "Train Epoch: 1 [22400/60000 (37%)]\tLoss: 0.389580\n",
      "Train Epoch: 1 [23040/60000 (38%)]\tLoss: 0.410104\n",
      "Train Epoch: 1 [23680/60000 (39%)]\tLoss: 0.375757\n",
      "Train Epoch: 1 [24320/60000 (41%)]\tLoss: 0.248711\n",
      "Train Epoch: 1 [24960/60000 (42%)]\tLoss: 0.330938\n",
      "Train Epoch: 1 [25600/60000 (43%)]\tLoss: 0.381253\n",
      "Train Epoch: 1 [26240/60000 (44%)]\tLoss: 0.420914\n",
      "Train Epoch: 1 [26880/60000 (45%)]\tLoss: 0.451814\n",
      "Train Epoch: 1 [27520/60000 (46%)]\tLoss: 0.433474\n",
      "Train Epoch: 1 [28160/60000 (47%)]\tLoss: 0.485042\n",
      "Train Epoch: 1 [28800/60000 (48%)]\tLoss: 0.360885\n",
      "Train Epoch: 1 [29440/60000 (49%)]\tLoss: 0.442029\n",
      "Train Epoch: 1 [30080/60000 (50%)]\tLoss: 0.346591\n",
      "Train Epoch: 1 [30720/60000 (51%)]\tLoss: 0.340833\n",
      "Train Epoch: 1 [31360/60000 (52%)]\tLoss: 0.371055\n",
      "Train Epoch: 1 [32000/60000 (53%)]\tLoss: 0.388851\n",
      "Train Epoch: 1 [32640/60000 (54%)]\tLoss: 0.273014\n",
      "Train Epoch: 1 [33280/60000 (55%)]\tLoss: 0.374838\n",
      "Train Epoch: 1 [33920/60000 (57%)]\tLoss: 0.298225\n",
      "Train Epoch: 1 [34560/60000 (58%)]\tLoss: 0.280861\n",
      "Train Epoch: 1 [35200/60000 (59%)]\tLoss: 0.201827\n",
      "Train Epoch: 1 [35840/60000 (60%)]\tLoss: 0.599721\n",
      "Train Epoch: 1 [36480/60000 (61%)]\tLoss: 0.296765\n",
      "Train Epoch: 1 [37120/60000 (62%)]\tLoss: 0.352147\n",
      "Train Epoch: 1 [37760/60000 (63%)]\tLoss: 0.349022\n",
      "Train Epoch: 1 [38400/60000 (64%)]\tLoss: 0.279511\n",
      "Train Epoch: 1 [39040/60000 (65%)]\tLoss: 0.136488\n",
      "Train Epoch: 1 [39680/60000 (66%)]\tLoss: 0.184672\n",
      "Train Epoch: 1 [40320/60000 (67%)]\tLoss: 0.360903\n",
      "Train Epoch: 1 [40960/60000 (68%)]\tLoss: 0.253812\n",
      "Train Epoch: 1 [41600/60000 (69%)]\tLoss: 0.191584\n",
      "Train Epoch: 1 [42240/60000 (70%)]\tLoss: 0.236678\n",
      "Train Epoch: 1 [42880/60000 (71%)]\tLoss: 0.323988\n",
      "Train Epoch: 1 [43520/60000 (72%)]\tLoss: 0.341968\n",
      "Train Epoch: 1 [44160/60000 (74%)]\tLoss: 0.357608\n",
      "Train Epoch: 1 [44800/60000 (75%)]\tLoss: 0.188344\n",
      "Train Epoch: 1 [45440/60000 (76%)]\tLoss: 0.261670\n",
      "Train Epoch: 1 [46080/60000 (77%)]\tLoss: 0.185893\n",
      "Train Epoch: 1 [46720/60000 (78%)]\tLoss: 0.296560\n",
      "Train Epoch: 1 [47360/60000 (79%)]\tLoss: 0.299167\n",
      "Train Epoch: 1 [48000/60000 (80%)]\tLoss: 0.344690\n",
      "Train Epoch: 1 [48640/60000 (81%)]\tLoss: 0.131952\n",
      "Train Epoch: 1 [49280/60000 (82%)]\tLoss: 0.345214\n",
      "Train Epoch: 1 [49920/60000 (83%)]\tLoss: 0.244834\n",
      "Train Epoch: 1 [50560/60000 (84%)]\tLoss: 0.241577\n",
      "Train Epoch: 1 [51200/60000 (85%)]\tLoss: 0.465652\n",
      "Train Epoch: 1 [51840/60000 (86%)]\tLoss: 0.399873\n",
      "Train Epoch: 1 [52480/60000 (87%)]\tLoss: 0.307516\n",
      "Train Epoch: 1 [53120/60000 (88%)]\tLoss: 0.385808\n",
      "Train Epoch: 1 [53760/60000 (90%)]\tLoss: 0.273694\n",
      "Train Epoch: 1 [54400/60000 (91%)]\tLoss: 0.208102\n",
      "Train Epoch: 1 [55040/60000 (92%)]\tLoss: 0.433666\n",
      "Train Epoch: 1 [55680/60000 (93%)]\tLoss: 0.205705\n",
      "Train Epoch: 1 [56320/60000 (94%)]\tLoss: 0.426417\n",
      "Train Epoch: 1 [56960/60000 (95%)]\tLoss: 0.210944\n",
      "Train Epoch: 1 [57600/60000 (96%)]\tLoss: 0.232607\n",
      "Train Epoch: 1 [58240/60000 (97%)]\tLoss: 0.399447\n",
      "Train Epoch: 1 [58880/60000 (98%)]\tLoss: 0.243570\n",
      "Train Epoch: 1 [59520/60000 (99%)]\tLoss: 0.338109\n"
     ]
    }
   ],
   "source": [
    "\n",
    "def train(epoch):\n",
    "    network.train()\n",
    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
    "        data = data.reshape(-1,28*28)\n",
    "        optimizer.zero_grad()\n",
    "        output = network(data)\n",
    "        loss = F.nll_loss(output, target)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        if batch_idx % log_interval == 0:\n",
    "            print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n",
    "                epoch, batch_idx * len(data), len(train_loader.dataset),\n",
    "                       100. * batch_idx / len(train_loader), loss.item()))\n",
    "            train_losses.append(loss.item())\n",
    "            train_counter.append(\n",
    "                (batch_idx * 64) + ((epoch - 1) * len(train_loader.dataset)))\n",
    "            torch.save(network.state_dict(), './model.pth')\n",
    "            torch.save(optimizer.state_dict(), './optimizer.pth')\n",
    "\n",
    "train(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-32-3c5581560232>:18: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  return F.log_softmax(out)\n",
      "/home/sychen/venv_envirment/tf_venv/lib/python3.8/site-packages/torch/nn/_reduction.py:44: UserWarning: size_average and reduce args will be deprecated, please use reduction='sum' instead.\n",
      "  warnings.warn(warning.format(ret))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test set: Avg. loss: 0.2548, Accuracy: 9242/10000 (92%)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "\n",
    "def test():\n",
    "    network.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    with torch.no_grad():\n",
    "        for data, target in test_loader:\n",
    "            data = data.reshape(-1,28*28)\n",
    "            output = network(data)\n",
    "            test_loss += F.nll_loss(output, target, size_average=False).item()\n",
    "            pred = output.data.max(1, keepdim=True)[1]\n",
    "            correct += pred.eq(target.data.view_as(pred)).sum()\n",
    "    test_loss /= len(test_loader.dataset)\n",
    "    test_losses.append(test_loss)\n",
    "    print('\\nTest set: Avg. loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\\n'.format(\n",
    "        test_loss, correct, len(test_loader.dataset),\n",
    "        100. * correct / len(test_loader.dataset)))\n",
    "\n",
    "\n",
    "test()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 结论\n",
    "CNN在参数相同的情况下  优秀于前馈"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "tf_venv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
